//
// (C) Copyright 2009 Irantha Suwandarathna (irantha@gmail.com)
// All rights reserved.
//

/* Copyright (c) 1995-2000, The Hypersonic SQL Group.
 * All rights reserved.
 *
 * Redistribution and use _in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions _in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer _in the documentation
 * and/or other materials provided with the distribution.
 *
 * Neither the name of the Hypersonic SQL Group nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE HYPERSONIC SQL GROUP,
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * This software consists of voluntary contributions made by many individuals
 * on behalf of the Hypersonic SQL Group.
 *
 *
 * For work added by the HSQL Development Group:
 *
 * Copyright (c) 2001-2008, The HSQL Development Group
 * All rights reserved.
 *
 * Redistribution and use _in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions _in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer _in the documentation
 * and/or other materials provided with the distribution.
 *
 * Neither the name of the HSQL Development Group nor the names of its
 * contributors may be used to endorse or promote products derived from this
 * software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL HSQL DEVELOPMENT GROUP, HSQLDB.ORG,
 * OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */


using System;
using System.Collections.Generic;


namespace EffiProzDB
{

    // fredt@users 20020218 - patch 455785 by hjbusch@users - large DECIMAL inserts
    // also Long.MIM_VALUE (bug 473388) inserts - applied to different parts
    // fredt@users 20020408 - patch 1.7.0 by fredt - exact integral types
    // integral values are cast into the smallest type that can hold them
    // fredt@users 20020501 - patch 550970 by boucherb@users - fewer StringBuilders
    // fredt@users 20020611 - patch 1.7.0 by fredt - correct statement logging
    // changes to the working of getLastPart() to return the correct statement for
    // logging _in the .script file.
    // also restructuring to reduce use of objects and speed up tokenising of
    // strings and quoted identifiers
    // fredt@users 20021112 - patch 1.7.2 by Nitin Chauhan - use of switch
    // rewrite of the majority of multiple if(){}else{} chains with switch(){}
    // fredt@users 20030610 - patch 1.7.2 - no StringBuilders

    /**
     * Provides the ability to tokenize SQL character sequences.
     *
     * Extensively rewritten and extended _in successive versions of HSQLDB.
     *
     * @author Thomas Mueller (Hypersonic SQL Group)
     * @version 1.8.0
     * @since Hypersonic SQL
     */


    public class Tokenizer
    {

        private  const int NO_TYPE = 0,
                                 NAME = 1,
                                 LONG_NAME = 2,
                                 SPECIAL = 3,
                                 NUMBER = 4,
                                 FLOAT = 5,
                                 STRING = 6,
                                 LONG = 7,
                                 DECIMAL = 8,
                                 BOOLEAN = 9,
                                 DATE = 10,
                                 TIME = 11,
                                 TIMESTAMP = 12,
                                 NULL = 13,
                                 NAMED_PARAM = 14;

        // used only internally
        private const  int QUOTED_IDENTIFIER = 15,
                                 REMARK_LINE = 16,
                                 REMARK = 17;
        private string sCommand;
        private int iLength;
        private int iIndex;
        private int tokenIndex;
        private int nextTokenIndex;
        private int beginIndex;
        private int iType;
        private string sToken;
        private int indexLongNameFirst = -1;
        private string sLongNameFirst = null;
        private int typeLongNameFirst;

        // getToken() will clear LongNameFirst unless retainFirst is set.
        private bool retainFirst = false;

        //    private string           sLongNameLast;
        // WAIT.  Don't do anything before popping another Token (because the
        // state variables aren't set properly due to a call of wait()).
        private bool bWait;
        private bool lastTokenQuotedID;

        // literals that are values
        static Dictionary<string,int> valueTokens;

        static Tokenizer()
        {
            valueTokens = new Dictionary<string,int>(StringComparer.InvariantCultureIgnoreCase);

            valueTokens.Add(Token.T_NULL, NULL);
            valueTokens.Add(Token.T_TRUE, BOOLEAN);
            valueTokens.Add(Token.T_FALSE, BOOLEAN);
        }

        public Tokenizer() { }

        public Tokenizer(string s)
        {

            sCommand = s;
            iLength = s.Length;
            iIndex = 0;
        }

        public void reset(string s)
        {

            sCommand = s;
            iLength = s.Length;
            iIndex = 0;
            tokenIndex = 0;
            nextTokenIndex = 0;
            beginIndex = 0;
            iType = NO_TYPE;
            typeLongNameFirst = NO_TYPE;
            sToken = null;
            indexLongNameFirst = -1;
            sLongNameFirst = null;

            //        sLongNameLast  = null;
            bWait = false;
            lastTokenQuotedID = false;
            retainFirst = false;
        }

        /**
         *
         * @ 
         */
        public void back()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when in Wait mode");
            }

            nextTokenIndex = iIndex;
            iIndex = (indexLongNameFirst != -1) ? indexLongNameFirst : tokenIndex;
            bWait = true;
        }

        /**
         * get the given token or throw
         *
         * for commands and simple unquoted identifiers only
         *
         * @param match
         *
         * @ 
         */
        public string getThis(string match)
        {

            getToken();
            matchThis(match);

            return sToken;
        }

        /**
         * for commands and simple unquoted identifiers only
         */
       public void matchThis(string match)
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when in Wait mode");
            }

            if (!sToken.Equals(match) || iType == QUOTED_IDENTIFIER
                    || iType == LONG_NAME)
            {
                string token = iType == LONG_NAME ? sLongNameFirst
                                                  : sToken;

                throw Trace.error(Trace.UNEXPECTED_TOKEN, Trace.TOKEN_REQUIRED,
                                  new Object[] {
                        token, match
                    });
            }
        }

       public  void throwUnexpected()
        {

            string token = iType == LONG_NAME ? sLongNameFirst
                                              : sToken;

            throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
        }

        /**
         * Used for commands only
         *
         *
         * @param match
         */
        public bool isGetThis(string match)
        {

            getToken();

            if (iType != QUOTED_IDENTIFIER && iType != LONG_NAME
                    && sToken.Equals(match))
            {
                return true;
            }

            back();

            return false;
        }

        /**
         * this methode is called before other wasXXX methods and takes
         * precedence
         */
        public bool wasValue()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when in Wait mode");
            }

            switch (iType)
            {

                case STRING:
                case NUMBER:
                case LONG:
                case FLOAT:
                case DECIMAL:
                case BOOLEAN:
                case NULL:
                    return true;

                default:
                    return false;
            }
        }

        public bool wasQuotedIdentifier()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when in Wait mode");
            }

            return lastTokenQuotedID;

            // iType won't help for LONG_NAMEs.
            //return iType == QUOTED_IDENTIFIER;
        }

        public bool wasFirstQuotedIdentifier()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when _in Wait mode");
            }

            return (typeLongNameFirst == QUOTED_IDENTIFIER);
        }

        /**
         * Method declaration
         *
         *
         * @return
         */
        public bool wasLongName()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when _in Wait mode");
            }

            return iType == LONG_NAME;
        }

        /**
         * Simple Name means a quoted or unquoted identifier without
         * qualifiers provided it is not _in the hKeyword list.
         *
         * @return
         */
       public bool wasSimpleName()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when _in Wait mode");
            }

            if (iType == QUOTED_IDENTIFIER && sToken.Length != 0)
            {
                return true;
            }

            if (iType != NAME)
            {
                return false;
            }

            return !Token.isKeyword(sToken);
        }

        /**
         * checks whether the previously obtained token was a (named) parameter
         *
         * @return true if the previously obtained token was a (named) parameter
         */
       public bool wasParameter()
        {

            Trace.doAssert(!bWait, "Querying state when _in Wait mode");

            return (iType == NAMED_PARAM);
        }

        /**
         * Name means all quoted and unquoted identifiers plus any word not _in the
         * hKeyword list.
         *
         * @return true if it's a name
         */
        public bool wasName()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when _in Wait mode");
            }

            if (iType == QUOTED_IDENTIFIER)
            {
                return true;
            }

            if (iType != NAME && iType != LONG_NAME)
            {
                return false;
            }

            return !Token.isKeyword(sToken);
        }

        public string getLongNamePre()
        {
            return null;
        }

        /**
         * Return first part of long name
         *
         *
         * @return
         */
        public string getLongNameFirst()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when _in Wait mode");
            }

            return sLongNameFirst;
        }

        public bool wasSimpleToken()
        {
            return iType != QUOTED_IDENTIFIER && iType != LONG_NAME
                   && iType != STRING && iType != NAMED_PARAM;
        }

        public string getSimpleToken()
        {

            getToken();

            if (!wasSimpleToken())
            {
                string token = iType == LONG_NAME ? sLongNameFirst
                                                  : sToken;

                throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
            }

            return sToken;
        }

        public bool wasThis(string match)
        {

            if (sToken.Equals(match) && iType != QUOTED_IDENTIFIER
                    && iType != LONG_NAME && iType != STRING)
            {
                return true;
            }

            return false;
        }

        /**
         * getName() is more broad than getSimpleName() _in that it includes
         * 2-part names as well
         *
         * @return popped name
         * @  if next token is not an AName
         */
        public string getName()
        {

            getToken();

            if (!wasName())
            {
                throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
            }

            return sToken;
        }

        /**
         * Returns a single, unqualified name (identifier)
         *
         * @return name
         * @ 
         */
        public string getSimpleName()
        {

            getToken();

            if (!wasSimpleName())
            {
                string token = iType == LONG_NAME ? sLongNameFirst
                                                  : sToken;

                throw Trace.error(Trace.UNEXPECTED_TOKEN, token);
            }

            return sToken;
        }

        /**
         * Return any token.
         *
         *
         * @return
         *
         * @ 
         */
        public string getString()
        {

            getToken();

            return sToken;
        }

        public int getInt()
        {

            long v = getBigint();

            if (v > int.MaxValue || v < int.MinValue)
            {
                throw Trace.error(Trace.WRONG_DATA_TYPE,
                                  Types.getTypeString(getType()));
            }

            return (int)v;
        }


        public static Decimal LONG_MAX_VALUE_INCREMENT =
            (Decimal.MinusOne);

        public long getBigint()
        {

            bool minus = false;

            getToken();

            if (sToken.Equals("-"))
            {
                minus = true;

                getToken();
            }

            Object o = getAsValue();
            int t = getType();

            switch (t)
            {

                case Types.INTEGER:
                    o = (long)(int)o;
                    break;
                case Types.BIGINT:
                    break;

                case Types.DECIMAL:

                    // only long.MaxValue + 1 together with minus is acceptable
                    if (minus && LONG_MAX_VALUE_INCREMENT.Equals(o))
                    {
                        return long.MinValue;
                    }
                    break;
                default:
                    throw Trace.error(Trace.WRONG_DATA_TYPE,
                                      Types.getTypeString(t));
            }

            long v = ((long)o);

            return minus ? -v
                         : v;
        }

        public Object getInType(int type)
        {

            getToken();

            Object o = getAsValue();
            int t = getType();

            if (t != type)
            {
                throw Trace.error(Trace.WRONG_DATA_TYPE, Types.getTypeString(t));
            }

            return o;
        }

        /**
         *
         *
         *
         * @return
         */
        public int getType()
        {

            if (bWait)
            {
                Trace.doAssert(false, "Querying state when _in Wait mode");
            }

            // todo: make sure it's used only for Values!
            // todo: synchronize iType with hColumn
            switch (iType)
            {

                case STRING:
                    return Types.VARCHAR;

                case NUMBER:
                    return Types.INTEGER;

                case LONG:
                    return Types.BIGINT;

                case FLOAT:
                    return Types.DOUBLE;

                case DECIMAL:
                    return Types.DECIMAL;

                case BOOLEAN:
                    return Types.BOOLEAN;

                case DATE:
                    return Types.DATE;

                case TIME:
                    return Types.TIME;

                case TIMESTAMP:
                    return Types.TIMESTAMP;

                default:
                    return Types.NULL;
            }
        }

        /**
         * Method declaration
         *
         *
         * @return
         *
         * @ 
         */
        public Object getAsValue()
        {

            if (!wasValue())
            {
                throw Trace.error(Trace.UNEXPECTED_TOKEN, sToken);
            }

            switch (iType)
            {

                case NULL:
                    return null;

                case STRING:

                    //fredt - no longer returning string with a singlequote as last char
                    return sToken;

                case LONG:
                    return long.Parse(sToken);

                case NUMBER:

                    // fredt - this returns unsigned values which are later negated.
                    // as a result int.MinValue or Long.MinValue are promoted
                    // to a wider type.
                    if (sToken.Length < 11)
                    {
                        try
                        {
                            return int.Parse(sToken);
                        }
                        catch (Exception ) { }
                    }

                    if (sToken.Length < 20)
                    {
                        try
                        {
                            iType = LONG;

                            return long.Parse(sToken);
                        }
                        catch (Exception) { }
                    }

                    iType = DECIMAL;

                    return Decimal.Parse(sToken);

                case FLOAT:
                    return Single.Parse(sToken);
                //long   l = BitConverter.DoubleToInt64Bits(d);

                //return ValuePool.getDouble(l);

                case DECIMAL:
                    return Decimal.Parse(sToken);

                case BOOLEAN:
                    return sToken.Equals("TRUE", StringComparison.InvariantCultureIgnoreCase) ? true
                                                           : false;

                case DATE:
                    return sToken;

                case TIME:
                    return sToken;

                case TIMESTAMP:
                    return sToken;

                default:
                    return sToken;
            }
        }

        /**
         * return the current position to be used for VIEW processing
         *
         * @return
         */
        public int getPosition()
        {
            return iIndex;
        }

        /**
         * mark the current position to be used for future getLastPart() calls
         *
         * @return
         */
        public string getPart(int begin, int end)
        {
            return sCommand.Substring(begin, end - (begin));
        }

        /**
         * mark the current position to be used for future getLastPart() calls
         *
         * @return
         */
        public int getPartMarker()
        {
            return beginIndex;
        }

        /**
         * mark the current position to be used for future getLastPart() calls
         *
         */
        public void setPartMarker()
        {
            beginIndex = iIndex;
        }

        /**
         * mark the position to be used for future getLastPart() calls
         *
         */
        public void setPartMarker(int position)
        {
            beginIndex = position;
        }

        /**
         * return part of the command string from the last marked position
         *
         * @return
         */
        public string getLastPart()
        {
            return sCommand.Substring(beginIndex, iIndex - (beginIndex));
        }

        // fredt@users 20020910 - patch 1.7.1 by Nitin Chauhan - rewrite as switch

        /**
         * Method declaration
         *
         *
         * @ 
         */
        private void getToken()
        {

            if (bWait)
            {
                bWait = false;
                iIndex = nextTokenIndex;

                return;
            }

            if (!retainFirst)
            {
                sLongNameFirst = null;
                indexLongNameFirst = -1;
                typeLongNameFirst = NO_TYPE;
            }

            while (iIndex < iLength
                    && Char.IsWhiteSpace(sCommand[(iIndex)]))
            {
                iIndex++;
            }

            sToken = "";
            tokenIndex = iIndex;

            if (iIndex >= iLength)
            {
                iType = NO_TYPE;

                return;
            }

            char c = sCommand[iIndex];
            bool point = false,
                    digit = false,
                    exp = false,
                    afterexp = false;
            bool end = false;
            char cfirst = '\u0000';

            lastTokenQuotedID = false;

            if (Char.IsLetter(c))
            {
                iType = NAME;
            }
            else if (Char.IsDigit(c))
            {
                iType = NUMBER;
                digit = true;
            }
            else
            {
                switch (c)
                {

                    case '(':
                        sToken = Token.T_OPENBRACKET;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case ')':
                        sToken = Token.T_CLOSEBRACKET;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case ',':
                        sToken = Token.T_COMMA;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case '*':
                        sToken = Token.T_MULTIPLY;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case '=':
                        sToken = Token.T_EQUALS;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case ';':
                        sToken = Token.T_SEMICOLON;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case '+':
                        sToken = Token.T_PLUS;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case '%':
                        sToken = Token.T_PERCENT;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case '?':
                        sToken = Token.T_QUESTION;
                        iType = SPECIAL;

                        iIndex++;

                        return;

                    case '@':
                    case ':':
                        Trace.check(++iIndex < iLength,
                                    Trace.UNEXPECTED_END_OF_COMMAND);

                        c = sCommand[(iIndex)];

                        Trace.check(Char.IsLetter(c),
                                    Trace.INVALID_IDENTIFIER, ":" + c);

                        iType = NAMED_PARAM;
                        break;

                    case '\"':
                        lastTokenQuotedID = true;
                        iType = QUOTED_IDENTIFIER;

                        iIndex++;

                        sToken = getString('"');

                        if (iIndex == sCommand.Length)
                        {
                            return;
                        }

                        c = sCommand[iIndex];

                        if (c == '.')
                        {
                            sLongNameFirst = sToken;
                            indexLongNameFirst = tokenIndex;
                            typeLongNameFirst = iType;

                            iIndex++;

                            if (retainFirst)
                            {
                                throw Trace.error(Trace.THREE_PART_IDENTIFIER);
                            }

                            // fredt - todo - avoid recursion - this has problems when there is whitespace
                            // after the dot - the same with NAME
                            retainFirst = true;

                            getToken();

                            retainFirst = false;
                            iType = LONG_NAME;
                        }

                        return;

                    case '\'':
                        iType = STRING;

                        iIndex++;

                        sToken = getString('\'');

                        return;

                    case '!':
                    case '<':
                    case '>':
                    case '|':
                    case '/':
                    case '-':
                        cfirst = c;
                        iType = SPECIAL;
                        break;

                    case '.':
                        iType = DECIMAL;
                        point = true;
                        break;

                    default:
                        throw Trace.error(Trace.UNEXPECTED_TOKEN,
                                          c.ToString());
                }
            }

            int start = iIndex++;

            while (true)
            {
                if (iIndex >= iLength)
                {
                    c = ' ';
                    end = true;

                    Trace.check(iType != STRING && iType != QUOTED_IDENTIFIER,
                                Trace.UNEXPECTED_END_OF_COMMAND);
                }
                else
                {
                    c = sCommand[iIndex];
                }

                switch (iType)
                {

                    case NAMED_PARAM:
                    case NAME:
                        if (Char.IsLetterOrDigit(c) || c == '_')
                        {
                            break;
                        }
                    
                        // fredt - todo new char[] to back sToken
                        sToken =
                            sCommand.Substring(start,
                                               iIndex-start ).ToUpper();

                        // the following only for NAME, not for NAMED_PARAM
                        if (iType == NAMED_PARAM)
                        {
                            return;
                        }

                        if (c == '.')
                        {
                            typeLongNameFirst = iType;
                            sLongNameFirst = sToken;
                            indexLongNameFirst = tokenIndex;

                            iIndex++;

                            if (retainFirst)
                            {
                                throw Trace.error(Trace.THREE_PART_IDENTIFIER);
                            }

                            retainFirst = true;

                            getToken();    // todo: eliminate recursion

                            retainFirst = false;
                            iType = LONG_NAME;
                        }
                        else if (c == '(')
                        {

                            // it is a function call
                        }
                        else
                        {

                            // if in value list then it is a value
                            int type;
                            
                            if (valueTokens.TryGetValue(sToken, out type))
                            {                          
                                iType = type;
                            }
                        }

                        return;

                    case QUOTED_IDENTIFIER:
                    case STRING:

                        // shouldn't get here
                        break;

                    case REMARK:
                        if (end)
                        {

                            // unfinished remark
                            // maybe print error here
                            iType = NO_TYPE;

                            return;
                        }
                        else if (c == '*')
                        {
                            iIndex++;

                            if (iIndex < iLength
                                    && sCommand[(iIndex)] == '/')
                            {

                                // using recursion here
                                iIndex++;

                                getToken();

                                return;
                            }
                        }
                        break;

                    case REMARK_LINE:
                        if (end)
                        {
                            iType = NO_TYPE;

                            return;
                        }
                        else if (c == '\r' || c == '\n')
                        {

                            // using recursion here
                            getToken();

                            return;
                        }
                        break;

                    case SPECIAL:
                        if (c == '/' && cfirst == '/')
                        {
                            iType = REMARK_LINE;

                            break;
                        }
                        else if (c == '-' && cfirst == '-')
                        {
                            iType = REMARK_LINE;

                            break;
                        }
                        else if (c == '*' && cfirst == '/')
                        {
                            iType = REMARK;

                            break;
                        }
                        else if (c == '>' || c == '=' || c == '|')
                        {
                            break;
                        }

                        sToken = sCommand.Substring(start, iIndex - (start));

                        return;

                    case NUMBER:
                    case FLOAT:
                    case DECIMAL:
                        if (Char.IsDigit(c))
                        {
                            digit = true;
                        }
                        else if (c == '.')
                        {
                            iType = DECIMAL;

                            if (point)
                            {
                                throw Trace.error(Trace.UNEXPECTED_TOKEN, ".");
                            }

                            point = true;
                        }
                        else if (c == 'E' || c == 'e')
                        {
                            if (exp)
                            {
                                throw Trace.error(Trace.UNEXPECTED_TOKEN, "E");
                            }

                            // HJB-2001-08-2001 - now we are sure it's a float
                            iType = FLOAT;

                            // first character after exp may be + or -
                            afterexp = true;
                            point = true;
                            exp = true;
                        }
                        else if (c == '-' && afterexp)
                        {
                            afterexp = false;
                        }
                        else if (c == '+' && afterexp)
                        {
                            afterexp = false;
                        }
                        else
                        {
                            afterexp = false;

                            if (!digit)
                            {
                                if (point && start == iIndex - 1)
                                {
                                    sToken = ".";
                                    iType = SPECIAL;

                                    return;
                                }

                                throw Trace.error(Trace.UNEXPECTED_TOKEN,
                                                  c.ToString());
                            }

                            sToken = sCommand.Substring(start, iIndex - (start));

                            return;
                        }
                        break;
                }

                iIndex++;
            }
        }

        // fredt - strings are constructed from new char[] objects to avoid slack
        // because these strings might end up as part of internal data structures
        // or table elements.
        // we may consider using pools to avoid recreating the strings
        private string getString(char quoteChar)
        {

            try
            {
                int nextIndex = iIndex;
                bool quoteInside = false;

                for (; ; )
                {
                    nextIndex = sCommand.IndexOf(quoteChar, nextIndex);

                    if (nextIndex < 0)
                    {
                        throw Trace.error(Trace.UNEXPECTED_END_OF_COMMAND);
                    }

                    if (nextIndex < iLength - 1
                            && sCommand[nextIndex + 1] == quoteChar)
                    {
                        quoteInside = true;
                        nextIndex += 2;

                        continue;
                    }

                    break;
                }

                char[] chBuffer = sCommand.ToCharArray(iIndex, nextIndex -iIndex);

                int j = chBuffer.Length;

                if (quoteInside)
                {
                    j = 0;

                    // fredt - loop assumes all occurences of quoteChar are paired
                    // this has already been checked by the preprocessing loop
                    for (int i = 0; i < chBuffer.Length; i++, j++)
                    {
                        if (chBuffer[i] == quoteChar)
                        {
                            i++;
                        }

                        chBuffer[j] = chBuffer[i];
                    }
                }

                iIndex = ++nextIndex;

                return new String(chBuffer, 0, j);
            }
            catch (HsqlException e)
            {
                throw e;
            }
            catch (Exception e)
            {
                e.ToString();
            }

            return null;
        }

        /**
         * Method declaration
         *
         *
         * @return
         */
        public int getLength()
        {
            return iLength;
        }
    }
}

